
### demo 爬取百度小说的内容

import re
import os
import requests
import asyncio
import aiohttp
import aiofiles

appKey='1351550300'
bookid = '4345130413'
chapterPageNum = 2
headChapterListUrl = 'https://boxnovel.baidu.com/boxnovel/wiseapi/chapterList?'
# chapterListUrl = f'https://boxnovel.baidu.com/boxnovel/wiseapi/chapterList?bookid={bookid}&pageNum={chapterPageNum}&order=asc&site='

# 同步获章节信息
def fetchChapterList(headUrl, bookid, page):
    chapterListUrl = headUrl + 'bookid=' + bookid +'&pageNum=' + str(page) +'&order=asc&site='
    response = requests.get(url=chapterListUrl)
    chapterResult = response.json()
    chapters = chapterResult['data']['chapter']['chapterInfo']
    return chapters

# 异步下载单个章节内容并处理内容的方法
async def fetchChapterContent(chapter_url, chapter_title):
    # chapter_url like 'http://h5.17k.com/chapter/3064327/39119768.html'
    # request url like https://h5.17k.com/ck/book/3064327/chapter/39119768?subAllPrice=1&appKey=1351550300
    # 从 chapter_url中提取 3064327 和 39119768
    matchResult = re.match(r".*?/chapter/(?P<group1>\d+)?/(?P<group2>\d+)?.html", chapter_url)
    if (matchResult):
        matchGroups = matchResult.groups()
        requestContentUrl = f'https://h5.17k.com/ck/book/{str(matchGroups[0])}/chapter/{str(matchGroups[1])}?subAllPrice=1&appKey={appKey}'
        print(requestContentUrl)
        async with aiohttp.request('GET', requestContentUrl) as r:
            resp = await r.json()
            async with aiofiles.open(f'H:{os.path.sep}tmp{os.path.sep}chapter{os.path.sep}{chapter_title}', "w", encoding='utf-8') as fp:
                await fp.write(resp['data']['content'][0]['text'])

# 异步获取章节内容的生成tasks的主方法
async def fetchChatpers(list):
    tasks = []
    for item in list:
        chapter_title = item['chapter_title']
        chapter_url = item['chapter_url']
        tasks.append(asyncio.create_task(fetchChapterContent(chapter_url, chapter_title)))
    await asyncio.wait(tasks)

# 章节列表
chapters = fetchChapterList(headChapterListUrl, bookid, chapterPageNum)
# print(chapters)

# 处理章节内容
# asyncio.run(fetchChatpers(chapters[1:3])) # instead of by get_event_loop
asyncio.get_event_loop().run_until_complete(fetchChatpers(chapters))
